import scrapy
from ImgCrawl.items import ImgcrawlItem
from time import sleep


class WallhavenSpiderSpider(scrapy.Spider):
    name = 'wallhaven_spider'
    allowed_domains = ['wallhaven.cc']
    url = 'https://wallhaven.cc/toplist'
    start_urls = [url]

    def parse(self, response):
        img_list = response.xpath(
            "//li//a[@class='preview']/@href").getall()

        for i in range(4):
            for item in img_list:
                arr = item.split('/')
                mark = arr[-2]
                name = arr[-1]
                namePre = name[0:2]
                url = "https://{}.wallhaven.cc/full/{}/wallhaven-{}.jpg".format(
                    mark, namePre, name)

                imgItem = ImgcrawlItem(name, url)
                yield imgItem

            next_page = scrapy.Request(self.url+'?page='+str(i))
            sleep(4)
            yield next_page
